package com.cobra.rdd.operator.transform

import org.apache.spark.{SparkConf, SparkContext}

import java.text.SimpleDateFormat

//分区不变原理
object Spark11_RDD_Operator_Transform_GROUP_BY_Test {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setMaster("local[*]").setAppName("Spark11_RDD_Operator_Transform_GROUP_BY_Test")
    val sc = new SparkContext(sparkConf)
    //算子获取
    val rdd = sc.textFile("datas/apache.log")
    val timeRDD = rdd.map(line => {
      val datas = line.split(" ")
      val time = datas(3)
      val format = new SimpleDateFormat("dd/MM/yyyy:HH:mm:ss")
      val date = format.parse(time)
      val format1 = new SimpleDateFormat("HH")
      val hour = format1.format(date)
      (hour, 1)
    }).groupBy(_._1)
    timeRDD.map{
      case (hour,iter)=>{
        (hour,iter.size)
      }
    }.collect().foreach(println)
    sc.stop()
  }
}
